***************
*** Table 2 ***
***************
** This do.file produces the results for Table 2 with HMS data
** Part 1: inflation 
** Part 2: new goods/feenstra ratio

global db "D:\Dropbox\unequal_gains\main_data"
global resultspath "D:\Dropbox\unequal_gains\QJE revision plan\analysis\clean_results"




*****************************
*****************************
***** Part 1: INFLATION *****
*****************************
*****************************



* (1) across barcodes
use "$db/Important Price Datasets/inflation_income.dta", clear
foreach i in tornqvist_all {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
foreach i in tornqvist_all {
gen `i'_infl_diff=`i'_infl-`i'_infl[_n-1]
tab `i'_infl_diff
}
drop if missing(tornqvist_all_infl_diff)
keep tornqvist_all_infl_diff
rename tornqvist_all_infl_diff infl_diff
gen AggregationLevel="Barcodes"
save "$resultspath\Table2.dta", replace
* now get number of barcodes (keeping in our anlaysis dataset, not just continued barcodes)
use "$db/Important Price Datasets/price_index_household_income_2004_2004_modules_final.dta", clear
foreach i of numlist 2005(1)2015{
append using "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta"
}
keep upc upc_ver_uc 
duplicates drop
gen N=[_N]
keep N 
duplicates drop 
gen AggregationLevel="Barcodes"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

* (2) across modules by quality deciles (QM)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossQM.dta", replace

foreach i of numlist 2004(1)2014 {

display "Starting Tornqvist inflation difference for year `i', QM"

* first, compute spending shares in each module by quality deciles 
local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending
drop _merge

* bring in info on UPC price decile (within product module) [for continued products, taking the average price in both periods]:
merge m:1 upc upc_ver_uc using "$db/Important Lists/upc_brand_size_final.dta"
keep if _merge==3
drop _merge
sum size1_amount, d
drop if size1_amount<r(p1) | size1_amount>r(p99)
gen average_unit_price_adj=(average_unit_price_all+average_unit_price_all_old)/(2*size1_amount*multi)
* create quality ranks without weights
sort product_module_code average_unit_price_adj
bysort product_module_code: gen double temp=[_N]
bysort product_module_code: gen double temp2=[_n]
gen rank=temp2/temp*100
gen quality_rank=.
foreach r of numlist 1(1)10 {
replace quality_rank=`r' if rank<=`r'*10 & missing(quality_rank)
}

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
*sum price_ratio_all, d
*drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_module quality income_quintile: egen double total_spending_inc_QM=sum(average_unit_price*total_quantity)
bysort product_module quality income_quintile: egen double total_spending_inc_QM_old=sum(average_unit_price_old*total_quantity_old)

keep product_module quality income_quintile total_spending_inc_QM total_spending_inc_QM_old
duplicates drop

* second, bring to file with inflation by quality-modules
merge m:1 product_module quality using "$db/Important Price Datasets/inflation_QM_`i'_2yrs.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_QM)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_QM_old)
gen double share_inc_QM=total_spending_inc_QM/total_spending_inc
gen double share_inc_QM_old=total_spending_inc_QM_old/total_spending_inc_old
gen weight_inc_QM=1/2*(share_inc_QM+share_inc_QM_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossQM.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossQM.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossQM.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Modules by Quality Deciles"
append using "$resultspath\Table2.dta"
save "$resultspath\Table2.dta", replace
* now get number of categories 
use "$db/Important Price Datasets/inflation_QM_2004.dta", clear
foreach i of numlist 2004(1)2014 {
append using "$db/Important Price Datasets/inflation_QM_`i'.dta"
}
keep product_module_code quality_rank
duplicates drop
gen N=[_N]
keep N 
duplicates drop 
gen AggregationLevel="Product Modules by Quality Deciles"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace


** (3) Across Modules (M)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossM.dta", replace

foreach i of numlist 2004(1)2014 {

display "Starting Tornqvist inflation difference for year `i', M"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_module income_quintile: egen double total_spending_inc_M=sum(average_unit_price*total_quantity)
bysort product_module income_quintile: egen double total_spending_inc_M_old=sum(average_unit_price_old*total_quantity_old)

keep product_module income_quintile total_spending_inc_M total_spending_inc_M_old
duplicates drop
merge m:1 product_module using "$db/Important Price Datasets/inflation_M_`i'.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_M)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_M_old)
gen double share_inc_M=total_spending_inc_M/total_spending_inc
gen double share_inc_M_old=total_spending_inc_M_old/total_spending_inc_old
gen weight_inc_M=1/2*(share_inc_M+share_inc_M_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossM.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossM.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossM.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Modules"
append using "$resultspath\Table2.dta"
save "$resultspath\Table2.dta", replace
* now get number of categories 
use "$db/Important Price Datasets/inflation_M_2004.dta", clear
foreach i of numlist 2004(1)2014 {
append using "$db/Important Price Datasets/inflation_M_`i'.dta"
}
keep product_module_code 
duplicates drop
gen N=[_N]
keep N 
duplicates drop 
gen AggregationLevel="Product Modules"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

** (4) Across groups (G)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossG.dta", replace

foreach i of numlist 2004(1)2014 {

display "Starting Tornqvist inflation difference for year `i', G"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_group income_quintile: egen double total_spending_inc_G=sum(average_unit_price*total_quantity)
bysort product_group income_quintile: egen double total_spending_inc_G_old=sum(average_unit_price_old*total_quantity_old)

keep product_group income_quintile total_spending_inc_G total_spending_inc_G_old
duplicates drop
merge m:1 product_group using "$db/Important Price Datasets/inflation_G_`i'.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_G)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_G_old)
gen double share_inc_G=total_spending_inc_G/total_spending_inc
gen double share_inc_G_old=total_spending_inc_G_old/total_spending_inc_old
gen weight_inc_G=1/2*(share_inc_G+share_inc_G_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossG.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossG.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossG.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Groups"
append using "$resultspath\Table2.dta"
save "$resultspath\Table2.dta", replace
* now get number of categories 
use "$db/Important Price Datasets/inflation_G_2004.dta", clear
foreach i of numlist 2004(1)2014 {
append using "$db/Important Price Datasets/inflation_G_`i'.dta"
}
keep product_group_code
duplicates drop
gen N=[_N]
keep N 
duplicates drop 
gen AggregationLevel="Product Groups"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

** (5) Across departments (D)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossD.dta", replace

foreach i of numlist 2004(1)2014 {

display "Starting Tornqvist inflation difference for year `i', D"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort department income_quintile: egen double total_spending_inc_D=sum(average_unit_price*total_quantity)
bysort department income_quintile: egen double total_spending_inc_D_old=sum(average_unit_price_old*total_quantity_old)

keep department income_quintile total_spending_inc_D total_spending_inc_D_old
duplicates drop
merge m:1 department using "$db/Important Price Datasets/inflation_D_`i'.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_D)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_D_old)
gen double share_inc_D=total_spending_inc_D/total_spending_inc
gen double share_inc_D_old=total_spending_inc_D_old/total_spending_inc_old
gen weight_inc_D=1/2*(share_inc_D+share_inc_D_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossD.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossD.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossD.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Departments"
append using "$resultspath\Table2.dta"
save "$resultspath\Table2.dta", replace
* now get number of categories 
use "$db/Important Price Datasets/inflation_D_2004.dta", clear
foreach i of numlist 2004(1)2014 {
append using "$db/Important Price Datasets/inflation_D_`i'.dta"
}
keep department_code
duplicates drop
gen N=[_N]
keep N 
duplicates drop 
gen AggregationLevel="Departments"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

* finalize table
use "$resultspath\Table2.dta", clear
egen temp=max(infl_diff)
gen share=infl_diff/temp*100
drop temp
format infl %8.4fc
format share %8.1fc
gsort - N
save "$resultspath\Table2.dta", replace




*****************************
*****************************
***** Part 2: NEW GOODS *****
*****************************
*****************************

** Compute product variety inequality at different levels of aggregation based on spending shares 

** (1) Across barcodes: 
* This is by definition the most detailed level of aggregation, i.e. report the full results
use "$db/Important Price Datasets/new_exit_income_final.dta", clear
keep ssnp ssep income_quintile year
duplicates drop
br
tsset income_q year
gen ssep_final=L.ssep

drop if year==2004
order ssnp ssep_final
drop ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Barcodes"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace


** (2) Across QM:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/new_exit_income_acrossQM.dta", replace

* note that here we need to have data on following year to be able to build the price ratio
foreach i of numlist 2004(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename average_unit_price_all average_unit_price_all_old

* bring in info on UPC price decile (within product module):
merge m:1 upc upc_ver_uc using "$db/Important Lists/upc_brand_size_final.dta"
keep if _merge==3
drop _merge
sum size1_amount, d
drop if size1_amount<r(p1) | size1_amount>r(p99)
gen average_unit_price_adj=average_unit_price_all/(size1_amount*multi)
* create quality ranks without weights 
sort product_module_code average_unit_price_adj
bysort product_module_code: gen double temp=[_N]
bysort product_module_code: gen double temp2=[_n]
gen rank=temp2/temp*100
gen quality_rank=.
foreach r of numlist 1(1)10 {
replace quality_rank=`r' if rank<=`r'*10 & missing(quality_rank)
}

* compute overall spending shares on modules by quality rank
drop total_spending*
bysort product_module quality income_quintile: egen double total_spendingQM=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareQM=total_spendingQM/total_spending

* compute spending shares 
keep product_module quality shareQM income_quintile
duplicates drop
merge m:1 product_module quality using "$db/Important Price Datasets/new_exit_QM_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_QM=sum(ssnpQM*shareQM)
bysort income_quintile: egen ssep_avg_QM=sum(ssepQM*shareQM)
keep income_quintile *_avg_QM
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/new_exit_income_acrossQM.dta"
drop if missing(year)
save "$db/Important Price Datasets/new_exit_income_acrossQM.dta", replace
}

use "$db/Important Price Datasets/new_exit_income_acrossQM.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2004
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Modules by Quality Deciles"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

** (3) Across M:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/new_exit_income_acrossM.dta", replace

foreach i of numlist 2004(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on modules by quality rank
drop total_spending*
bysort product_module income_quintile: egen double total_spendingM=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareM=total_spendingM/total_spending

* compute spending shares 
keep product_module shareM income_quintile
duplicates drop
merge m:1 product_module using "$db/Important Price Datasets/new_exit_M_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_M=sum(ssnpM*shareM)
bysort income_quintile: egen ssep_avg_M=sum(ssepM*shareM)
keep income_quintile *_avg_M
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/new_exit_income_acrossM.dta"
drop if missing(year)
save "$db/Important Price Datasets/new_exit_income_acrossM.dta", replace
}

use "$db/Important Price Datasets/new_exit_income_acrossM.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2004
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Modules"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace


** (4) Across G:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/new_exit_income_acrossG.dta", replace

foreach i of numlist 2004(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on product groups
drop total_spending*
bysort product_group income_quintile: egen double total_spendingG=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareG=total_spendingG/total_spending

* compute spending shares 
keep product_group shareG income_quintile
duplicates drop
merge m:1 product_group using "$db/Important Price Datasets/new_exit_G_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_G=sum(ssnpG*shareG)
bysort income_quintile: egen ssep_avg_G=sum(ssepG*shareG)
keep income_quintile *_avg_G
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/new_exit_income_acrossG.dta"
drop if missing(year)
save "$db/Important Price Datasets/new_exit_income_acrossG.dta", replace
}

use "$db/Important Price Datasets/new_exit_income_acrossG.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2004
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Groups"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

** (5) Across D:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/new_exit_income_acrossD.dta", replace

foreach i of numlist 2004(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on departments
drop total_spending*
bysort department income_quintile: egen double total_spendingD=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareD=total_spendingD/total_spending

* compute spending shares 
keep department shareD income_quintile
duplicates drop
merge m:1 department_code using "$db/Important Price Datasets/new_exit_D_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_D=sum(ssnpD*shareD)
bysort income_quintile: egen ssep_avg_D=sum(ssepD*shareD)
keep income_quintile *_avg_D
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/new_exit_income_acrossD.dta"
drop if missing(year)
save "$db/Important Price Datasets/new_exit_income_acrossD.dta", replace
}

use "$db/Important Price Datasets/new_exit_income_acrossD.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2004
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Departments"
merge 1:1 Aggregation using "$resultspath\Table2.dta"
drop _merge
save "$resultspath\Table2.dta", replace

* finalize table 
use "$resultspath\Table2.dta", clear
egen temp=max(log_feenstra)
gen share_feenstra=log_feenstra/temp*100
drop temp
rename share share_infl
order AggregationLevel N infl_diff share_infl log_feenstra_diff share_f
gsort - N
format *_diff %8.3fc
format share* %8.1fc
save "$resultspath\Table2.dta", replace


